this notebook is structured in 4 parts
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import datetime
import pickle
import pandas as pd
from pandas.io.parsers import read_csv
import random
import imgaug.augmenters as iaa
import imgaug as ia
import cv2 as cv
print("cv version (tested on 4.5.1) is {}".format(cv.__version__))
import tensorflow as tf
print("tf version (tested on 2.4.1) is {}".format(tf.__version__))
from sklearn.utils import shuffle
import datetime
from tensorflow.keras import Model
from tensorflow.keras.models import Sequential, save_model, load_model
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D,Dropout
from tensorflow.keras.utils import to_categorical
import glob, os
from PIL import Image
notebook_root_path = os.getcwd()
print("current working directory : ", notebook_root_path)
cv version (tested on 4.5.1) is 4.5.1 tf version (tested on 2.4.1) is 2.4.1 current working directory : /Users/giorgio/Documents/Udacity_SDC/P3-LeNet_traffic_signs/P3-Traffic-Sign-Classifier
Since some processes and functions require quite some calculation time, I set up some flags that can be set to enable/disable some specific steps. The flags are all set to True before the submission.
plot_writeup_images = True
generate_fake_data_for_augmentation = True
train_model_flag = True
process_own_images = True
The pickled data is a dictionary with 4 key/value pairs:
'features' is a 4D array containing raw pixel data of the traffic sign images, (num examples, width, height, channels).'labels' is a 1D array containing the label/class id of the traffic sign. The file signnames.csv contains id -> name mappings for each id.'sizes' is a list containing tuples, (width, height) representing the original width and height the image.'coords' is a list containing tuples, (x1, y1, x2, y2) representing coordinates of a bounding box around the sign in the image. THESE COORDINATES ASSUME THE ORIGINAL IMAGE. THE PICKLED DATA BELOW CONTAINS RESIZED VERSIONS (32 by 32) OF THESE IMAGEStraining_file = '../traffic-signs-data/train.p'
validation_file= '../traffic-signs-data/valid.p'
testing_file = '../traffic-signs-data/test.p'
with open(training_file, mode='rb') as f:
train = pickle.load(f)
with open(validation_file, mode='rb') as f:
valid = pickle.load(f)
with open(testing_file, mode='rb') as f:
test = pickle.load(f)
X_train, y_train = train['features'], train['labels']
X_valid, y_valid = valid['features'], valid['labels']
X_test, y_test = test['features'], test['labels']
assert(len(X_train) == len(y_train))
assert(len(X_valid) == len(y_valid))
assert(len(X_test) == len(y_test))
def print_stats(x,y,text):
"""
Helper function print simple dataset stats
:param x: dataset features
:param y: dataset labels
:param text: simple text to print
:return: none
"""
print('Stats for the {} dataset'.format(text))
assert(len(x) == len(y))
# Number of training examples
size = len(x)
# What's the shape of a traffic sign image?
index = random.randint(0, len(x))
image = x[index]
image_shape = image.shape
# How many unique classes/labels there are in the dataset.
if(len(y.shape)>1):
n_classes = y.shape[1]
else:
n_classes = len(set(y))
print('---------------------')
print("Number of examples =", size)
print("Image data shape =", image_shape)
print("Number of classes =", n_classes)
print('---------------------')
print_stats(X_train,y_train,'Training')
print_stats(X_valid,y_valid,'Validation')
print_stats(X_test,y_test,'Testing')
Stats for the Training dataset --------------------- Number of examples = 34799 Image data shape = (32, 32, 3) Number of classes = 43 --------------------- Stats for the Validation dataset --------------------- Number of examples = 4410 Image data shape = (32, 32, 3) Number of classes = 43 --------------------- Stats for the Testing dataset --------------------- Number of examples = 12630 Image data shape = (32, 32, 3) Number of classes = 43 ---------------------
There are three visualization functions.
One 'plot_sample_of_each' shows a sublot of 15 (by detault) color images from the training dataset. I decided to show only one image per class. This method is used to produce a picture for the writeup.
The method 'plot_sign_count_bar' takes as input the signnames and the number of signs to produce a sorted bar plot that shows the distribution of the different classes in terms of number of images. This method is useful to check how augmentation techniques shape the distribution of images. Whe I apply this function to the original datasets (train, valid and test) it shows that they all have a similar distribution among the classes and they are not balanced.
the method 'plot_samples_from_all_classes' scrolls the entire datasets and it shows a few random images per each class (10 images per class by default). This method is useful to observe the datasets and to check the overall impact of the image filters.
def plot_sample_of_each(rows=3,columns=5):
"""
Plots a subplot with rows*columns samples of images from the training dataset
:return: saves a file
"""
sign_names = pd.read_csv('signnames.csv')
random_unique_sign_img = []
sign_name_title = []
df = pd.DataFrame({'sign':list(y_train)})
classes = list(df.sign.unique())
for sign in classes:
unique_sign = df[df['sign']==sign].sample(n=1)
random_unique_sign_img.append(X_train[unique_sign.index[0]])
title = sign_names[sign_names['ClassId']==sign]['SignName'].values[0]
if len(title.split())<5:
sign_name_title.append(title)
else:
two_l = title.split()[0:4]
two_l.extend('\n')
two_l.extend(title.split()[4:])
sign_name_title.append(' '.join(two_l))
f, (axes) = plt.subplots(rows,columns, figsize=(32, 32))
f.tight_layout()
for i in range(rows):
for j in range(columns):
if(len(random_unique_sign_img)>0):
axes[i][j].imshow(random_unique_sign_img.pop(0))
axes[i][j].set_title(sign_name_title.pop(0),fontsize=30)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.05,hspace = 0.4,wspace = 0.1)
f.savefig('writeup_images/Some_example_sings_{}x{}.png'.format(rows,columns),dpi=300, bbox_inches = "tight")
if(plot_writeup_images):
plot_sample_of_each(rows=3,columns=5)
def build_sign_dict(signnames,sign_classes,class_counts,labels):
"""
Creates and plots an horizontal bar plot
:param signnames: sorted list traffic signs
(index corresponds to the label)
:param sign_classes: indices of the first element in the label list.
used as filter mask
:param labels: dataset labels as (1D array)
:return sign_dict: dict with sign names and values
"""
#get the size of the list
size = len(signnames)
#prepare array
filtered_counter = np.zeros(size)
#get unique values
unique_lab = set(labels)
cnt = 0
for i in range(size):
if i in unique_lab:
filtered_counter[cnt] = class_counts[cnt]
cnt += 1
#filter the sign names with existing labels
filtered_signnames = signnames[sign_classes]
#put sign names and counters into a dict
sign_dict = {}
for s,c in zip(filtered_signnames,filtered_counter):
sign_dict[s]=c
return sign_dict
def plot_sign_count_bar(signnames,labels,dataset_name='Training'):
"""
Creates and plots an horizontal bar plot
:param signnames: sorted list traffic signs
(index corresponds to the label)
:param labels: list of labels, it can be in categorical form or 1D
:param dataset_name: string used as filename title
:return: saves a png file
"""
# if label is converted to categorical
# reverse it to 1D array
if (len(labels.shape)>1):
labels = np.argmax(labels,1)
#convert labels to int
labels = [np.int(i) for i in labels]
#sort labels
labels = np.sort(labels)
#get unique values and their position
sign_classes, class_indices, class_counts = np.unique(
labels,
return_index = True,
return_counts = True)
sign_dict = build_sign_dict(signnames,sign_classes,class_counts,labels)
#sort dict values for nicer visualization
values = sorted(sign_dict.values())
labels = sorted(sign_dict, key=sign_dict.get)
#generate figure
plt.rcdefaults()
fig, ax = plt.subplots()
plt.tight_layout()
fig.set_size_inches(15, 10.5)
plt.margins(y=0.01)
plt.grid(True)
ax.barh(range(len(sign_dict)), values, align='center')
bars = plt.barh(range(len(sign_dict)), values, align='center',color=['black','red'])
plt.yticks(range(len(sign_dict)), labels)
ax.invert_yaxis() # labels read top-to-bottom
ax.set_xlabel('Count')
fig.savefig('writeup_images/{}_data_distrib.png'.format(dataset_name),
dpi=300,
bbox_inches = "tight")
signnames = read_csv("signnames.csv").values[:, 1]
if(plot_writeup_images):
#visualizing and saving Training data distribution
plot_sign_count_bar(signnames,y_train,dataset_name='Training')
#visualizing and saving Validation data distribution
plot_sign_count_bar(signnames,y_valid,dataset_name='Validation')
#visualizing and saving Test data distribution
plot_sign_count_bar(signnames,y_test,dataset_name='Test')
def plot_samples_from_all_classes(images,labels,n_samples=10,color='rgb', show_indices=False):
"""
Plots some random examples per each class of images
:param images: list images
:param labels: list of numerical labels
:param n_samples: number of samples per class
:param color: default is rgb, valid alternative is gray
:return: none
"""
# if label is converted to categorical
# reverse it to 1D array
if (len(labels.shape)>1):
labels = np.argmax(labels,1)
#convert labels to int
labels = [np.int(i) for i in labels]
sign_classes, class_indices, class_counts = np.unique(labels,
return_index = True,
return_counts = True)
df = pd.DataFrame({'sign':list(labels)})
col_width = max(len(name) for name in signnames)
for c, c_index, c_count in zip(sign_classes, class_indices, class_counts):
print("Class %i: %-*s %s samples" % (c, col_width, signnames[c], str(c_count)))
fig = plt.figure(figsize = (6, 1))
fig.subplots_adjust(left = 0,
right = 1,
bottom = 0,
top = 1,
hspace = 0.05,
wspace = 0.05)
if (n_samples>c_count):
n_samples_show = c_count
else:
n_samples_show = n_samples
random_indices = df[df['sign']==c].sample(n=n_samples_show)
random_indices = random_indices.index[:]
for i in range(n_samples_show):
axis = fig.add_subplot(1, n_samples_show, i + 1, xticks=[], yticks=[])
if color == 'gray':
axis.imshow(images[random_indices[i]],cmap='gray')
else:
axis.imshow(images[random_indices[i]])
plt.show()
if(show_indices):
print("Image indices: {}".format(random_indices))
print("--------------------------------------------------------------------------------------\n")
if(plot_writeup_images):
plot_samples_from_all_classes(X_train,y_train,show_indices=True)
Class 0: Speed limit (20km/h) 180 samples
Image indices: Int64Index([10030, 10018, 10007, 9974, 10036, 9961, 10125, 10069, 10073,
10090],
dtype='int64')
--------------------------------------------------------------------------------------
Class 1: Speed limit (30km/h) 1980 samples
Image indices: Int64Index([3937, 3643, 2588, 2452, 3697, 2854, 2414, 2618, 3770, 3521], dtype='int64') -------------------------------------------------------------------------------------- Class 2: Speed limit (50km/h) 2010 samples
Image indices: Int64Index([32186, 32137, 32073, 33302, 32345, 33275, 32829, 32011, 32648,
33085],
dtype='int64')
--------------------------------------------------------------------------------------
Class 3: Speed limit (60km/h) 1260 samples
Image indices: Int64Index([5500, 5976, 5908, 5503, 6307, 5972, 6342, 6487, 5609, 6220], dtype='int64') -------------------------------------------------------------------------------------- Class 4: Speed limit (70km/h) 1770 samples
Image indices: Int64Index([7526, 6915, 7663, 8441, 7515, 7400, 8552, 7927, 8264, 7375], dtype='int64') -------------------------------------------------------------------------------------- Class 5: Speed limit (80km/h) 1650 samples
Image indices: Int64Index([12652, 12465, 13923, 13978, 12585, 13608, 12536, 12955, 12468,
12835],
dtype='int64')
--------------------------------------------------------------------------------------
Class 6: End of speed limit (80km/h) 360 samples
Image indices: Int64Index([21534, 21647, 21727, 21597, 21769, 21536, 21671, 21496, 21590,
21730],
dtype='int64')
--------------------------------------------------------------------------------------
Class 7: Speed limit (100km/h) 1290 samples
Image indices: Int64Index([24654, 23731, 24003, 24239, 24386, 24697, 24483, 24792, 24498,
23867],
dtype='int64')
--------------------------------------------------------------------------------------
Class 8: Speed limit (120km/h) 1260 samples
Image indices: Int64Index([16136, 15927, 16832, 16465, 16901, 16273, 16137, 16420, 16103,
16691],
dtype='int64')
--------------------------------------------------------------------------------------
Class 9: No passing 1320 samples
Image indices: Int64Index([11811, 11621, 12299, 11294, 11078, 11615, 12217, 11061, 11325,
12316],
dtype='int64')
--------------------------------------------------------------------------------------
Class 10: No passing for vehicles over 3.5 metric tons 1800 samples
Image indices: Int64Index([17796, 18667, 17677, 17199, 18617, 18295, 18356, 17983, 18869,
17588],
dtype='int64')
--------------------------------------------------------------------------------------
Class 11: Right-of-way at the next intersection 1170 samples
Image indices: Int64Index([8633, 9654, 9087, 9278, 9197, 9072, 8648, 8846, 8835, 8873], dtype='int64') -------------------------------------------------------------------------------------- Class 12: Priority road 1890 samples
Image indices: Int64Index([27541, 29097, 28070, 27885, 27735, 28506, 27530, 28646, 28649,
28254],
dtype='int64')
--------------------------------------------------------------------------------------
Class 13: Yield 1920 samples
Image indices: Int64Index([22006, 22304, 22727, 23370, 23330, 22064, 23028, 22438, 22396,
22983],
dtype='int64')
--------------------------------------------------------------------------------------
Class 14: Stop 690 samples
Image indices: Int64Index([29855, 29656, 29593, 29416, 29341, 29403, 29278, 29340, 29688,
29617],
dtype='int64')
--------------------------------------------------------------------------------------
Class 15: No vehicles 540 samples
Image indices: Int64Index([29926, 30055, 30087, 30303, 30024, 29973, 30064, 30121, 30302,
30031],
dtype='int64')
--------------------------------------------------------------------------------------
Class 16: Vehicles over 3.5 metric tons prohibited 360 samples
Image indices: Int64Index([5137, 5247, 5290, 5097, 5161, 5011, 5028, 5245, 5150, 5271], dtype='int64') -------------------------------------------------------------------------------------- Class 17: No entry 990 samples
Image indices: Int64Index([30746, 31065, 30741, 30668, 31212, 30782, 30865, 30810, 31252,
31420],
dtype='int64')
--------------------------------------------------------------------------------------
Class 18: General caution 1080 samples
Image indices: Int64Index([20617, 20714, 21413, 21071, 20531, 20519, 21242, 21225, 20903,
21335],
dtype='int64')
--------------------------------------------------------------------------------------
Class 19: Dangerous curve to the left 180 samples
Image indices: Int64Index([6633, 6783, 6694, 6725, 6675, 6696, 6770, 6702, 6754, 6739], dtype='int64') -------------------------------------------------------------------------------------- Class 20: Dangerous curve to the right 300 samples
Image indices: Int64Index([26051, 26009, 26013, 26216, 26143, 26054, 26174, 26076, 26154,
26133],
dtype='int64')
--------------------------------------------------------------------------------------
Class 21: Double curve 270 samples
Image indices: Int64Index([25720, 25827, 25701, 25724, 25812, 25933, 25811, 25852, 25711,
25947],
dtype='int64')
--------------------------------------------------------------------------------------
Class 22: Bumpy road 330 samples
Image indices: Int64Index([4826, 4765, 4651, 4649, 4657, 4723, 4749, 4697, 4714, 4551], dtype='int64') -------------------------------------------------------------------------------------- Class 23: Slippery road 450 samples
Image indices: Int64Index([1883, 1888, 1929, 2009, 1790, 1875, 2001, 2198, 2167, 1968], dtype='int64') -------------------------------------------------------------------------------------- Class 24: Road narrows on the right 240 samples
Image indices: Int64Index([10936, 10978, 10974, 10914, 11030, 10855, 10937, 11035, 10891,
10844],
dtype='int64')
--------------------------------------------------------------------------------------
Class 25: Road work 1350 samples
Image indices: Int64Index([34486, 34231, 34665, 33595, 33730, 33714, 33757, 34473, 33934,
33871],
dtype='int64')
--------------------------------------------------------------------------------------
Class 26: Traffic signals 540 samples
Image indices: Int64Index([1733, 1509, 1417, 1451, 1415, 1628, 1475, 1488, 1667, 1379], dtype='int64') -------------------------------------------------------------------------------------- Class 27: Pedestrians 210 samples
Image indices: Int64Index([10514, 10523, 10395, 10473, 10362, 10388, 10436, 10497, 10495,
10530],
dtype='int64')
--------------------------------------------------------------------------------------
Class 28: Children crossing 480 samples
Image indices: Int64Index([27025, 27088, 27294, 26882, 27259, 26945, 27246, 27281, 26859,
27252],
dtype='int64')
--------------------------------------------------------------------------------------
Class 29: Bicycles crossing 240 samples
Image indices: Int64Index([10637, 10716, 10664, 10711, 10602, 10665, 10599, 10570, 10650,
10563],
dtype='int64')
--------------------------------------------------------------------------------------
Class 30: Beware of ice/snow 390 samples
Image indices: Int64Index([25130, 25310, 25301, 25149, 25252, 25286, 25357, 25375, 25353,
25047],
dtype='int64')
--------------------------------------------------------------------------------------
Class 31: Wild animals crossing 690 samples
Image indices: Int64Index([372, 552, 800, 222, 219, 471, 572, 563, 518, 247], dtype='int64') -------------------------------------------------------------------------------------- Class 32: End of all speed and passing limits 210 samples
Image indices: Int64Index([10152, 10220, 10204, 10234, 10273, 10298, 10178, 10179, 10327,
10339],
dtype='int64')
--------------------------------------------------------------------------------------
Class 33: Turn right ahead 599 samples
Image indices: Int64Index([26783, 26834, 26473, 26809, 26654, 26268, 26763, 26421, 26450,
26666],
dtype='int64')
--------------------------------------------------------------------------------------
Class 34: Turn left ahead 360 samples
Image indices: Int64Index([20079, 20216, 20130, 20151, 20054, 20302, 20350, 20172, 20267,
20321],
dtype='int64')
--------------------------------------------------------------------------------------
Class 35: Ahead only 1080 samples
Image indices: Int64Index([19762, 19350, 18959, 18931, 19289, 19490, 19002, 19265, 19154,
19256],
dtype='int64')
--------------------------------------------------------------------------------------
Class 36: Go straight or right 330 samples
Image indices: Int64Index([1176, 902, 1073, 954, 1153, 1119, 929, 956, 1171, 1104], dtype='int64') -------------------------------------------------------------------------------------- Class 37: Go straight or left 180 samples
Image indices: Int64Index([4976, 4871, 4852, 4878, 4893, 4981, 4843, 4860, 4853, 4970], dtype='int64') -------------------------------------------------------------------------------------- Class 38: Keep right 1860 samples
Image indices: Int64Index([15612, 15171, 14779, 15231, 15836, 14510, 15227, 14953, 15109,
14772],
dtype='int64')
--------------------------------------------------------------------------------------
Class 39: Keep left 270 samples
Image indices: Int64Index([25586, 25629, 25511, 25497, 25501, 25485, 25538, 25514, 25619,
25632],
dtype='int64')
--------------------------------------------------------------------------------------
Class 40: Roundabout mandatory 300 samples
Image indices: Int64Index([4463, 4339, 4244, 4247, 4252, 4307, 4221, 4430, 4381, 4342], dtype='int64') -------------------------------------------------------------------------------------- Class 41: End of no passing 210 samples
Image indices: Int64Index([61, 20, 38, 189, 114, 74, 142, 37, 187, 148], dtype='int64') -------------------------------------------------------------------------------------- Class 42: End of no passing by vehicles over 3.5 metric tons 210 samples
Image indices: Int64Index([9875, 9937, 9783, 9753, 9956, 9915, 9874, 9852, 9921, 9951], dtype='int64') --------------------------------------------------------------------------------------
I designed and implemented a deep learning model that learns to recognize traffic signs. The model is trained and testd on the German Traffic Sign Dataset. The model is also tested against some additional images that I personally took in Germany.
The current model is a modified version of the the LeNet-5 implementation shown in the classroom.
The current implementation uses five hidden layers, three convolutional layers with increasing depth (I obtained the best results with depths=[32,64,128]) and two fully connected layers before the output layer.
To contrast the effects of overfitting, the model uses dropout for regularization only applied to the fully connected layers, that is layer 4 and layer 5 (and not to the convolutional layers). The best results (98.4% on validation accuracy and 96.1% on test accuracy) are obtained with p4=0.3 and p5=0.2
The network has a total number of trainable parameters equal to 244,851
The network architecture accepts a 32x32x1 image as input. The images have a single channel (we assume grayscale images)
Layer 1: Convolutional. with depth 32, strides 1x1 and padding 'valid'. The output shape is 30x30x32.
Activation. ReLU
Pooling. Max pooling with kernel size (2x2), padding='same' output shape is 15x15x32.
Layer 2: Convolutional. with depth 64, strides 1x1 and padding 'valid'. The output shape is 13x13x64.
Activation. ReLU
Pooling. Max pooling with kernel size (2x2), padding='same' output shape is 7x7x64.
Layer 3: Convolutional. with depth 128, strides 1x1 and padding 'valid'. The output shape is 5x5x128.
Activation. ReLU
Pooling. Max pooling with kernel size (2x2), padding='same' output shape is 3x3x128.
Flatten. Flatten the output shape of the final pooling layer results in 1152 elements.
Layer 4: Fully Connected. This should have 120 outputs.
Activation. ReLU
Regularizatin. Dropout with keep probability of 0.3
Layer 5: Fully Connected. This should have 84 outputs.
Activation. ReLU
Regularizatin. Dropout with keep probability of 0.2
Layer 5-out: Fully Connected (Logits). 43 outputs
Activation. Softmax
Return the result of the 2nd fully connected layer.
As suggested in the paper from Sermanet et al. I applied several image pre-processing techniques to improve the model performance. Some of the filters and techniques are also used to generate fake data that helps contrasting model overfitting.
def normalise_input(x,shift_for_plotting=False):
"""
Nromalize and center the distribution of image pixels
:param x: pixels
:param shift_for_plotting: to be able to plot the effect in a plot,
the distribution should be positive, hence shift from [-1,1] to [0,1]
:return: same image as input with normalised and centered pixel distrib.
"""
# convert from integers to floats
x = x.astype('float32')
# calculate global mean and standard deviation
mean, std = x.mean(), x.std()
print('Mean: %.3f, Standard Deviation: %.3f' % (mean, std))
# global standardization of pixels
x = (x - mean) / std
# clip pixel values to [-1,1]
x = np.clip(x, -1.0, 1.0)
if(shift_for_plotting):
# shift from [-1,1] to [0,1] with 0.5 mean
output = (x + 1.0) / 2.0
else:
output = x
# confirm it had the desired effect
mean, std = output.mean(), output.std()
print('New Mean: %.3f, Standard Deviation: %.3f' % (mean, std))
assert(x.shape==output.shape)
return output
def image_crop(img,percent,pad_cval):
"""
Applies the scale filter to the input image and it returns a copy
:param image: input image
:param percent: The number of pixels to crop (negative values) or pad (positive values)
on each side of the image given as a *fraction* of the image
height/width.
:param pad_cval: a tuple ``(a, b)``, then a value will be uniformly sampled
per image from the interval ``[a, b]``
:return: copy of the image after processing
"""
aug = iaa.CropAndPad(percent=percent,pad_cval=pad_cval)
return aug(image=img)
def image_scale(img,scale_dict):
"""
Applies the scale filter to the input image and it returns a copy
:param image: input image
:param scale_dict:
{"x": number/tuple/list/StochasticParameter,
"y": number/tuple/list/StochasticParameter}
:return: copy of the image after processing
"""
aug = iaa.Affine(scale=scale_dict)
return aug(image=img)
def image_translate(img,translate_dict):
"""
Applies the translation filter to the input image and it returns a copy
:param image: input image
:param translate_dict:
{"x": number/tuple/list/StochasticParameter,
"y": number/tuple/list/StochasticParameter}
:return: copy of the image after processing
"""
aug = iaa.Affine(translate_percent=translate_dict)
return aug(image=img)
def image_rotate(img,rotate_tuple):
"""
Applies the translation filter to the input image and it returns a copy
:param image: input image
:param rotate_tuple: a tuple ``(a, b)``, then a value will be uniformly sampled
per image from the interval ``[a, b]`` and used as the rotation
value.
:return: copy of the image after processing
"""
aug = iaa.Affine(rotate=rotate_tuple)
return aug(image=img)
def image_shear(img,shear_tuple):
"""
Applies the translation filter to the input image and it returns a copy
:param image: input image
:param shear_tuple: a tuple ``(a, b)``, then a value will be uniformly sampled
per image from the interval ``[a, b]`` and used as the shear
value.
:return: copy of the image after processing
"""
aug = iaa.Affine(shear=shear_tuple)
return aug(image=img)
def applyCLAHE(img ):
"""
Applies the CLAHE histogram equalization
ref: https://docs.opencv.org/master/d5/daf/tutorial_py_histogram_equalization.html
:param image: input image
:return: copy of the image after processing
"""
clahe = cv.createCLAHE()
return clahe.apply(img)
def plot_writeup_preprocess_images():
# choose one image
index = 553
image = X_train[index]
#grayscale
grayimg = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
#apply clahe
clahe = applyCLAHE(grayimg)
#scale
scaled = image_scale(grayimg,{"x": (0.5), "y": (0.5)})
#tranlate
translated = image_translate(grayimg,{"x": (0.2), "y": (0.2)})
#rotate
rotated = image_rotate(grayimg,(45))
#shear
sheared = image_shear(grayimg,(20))
#crop
cropped = image_crop(grayimg,(0.1),(0, 255))
#normalise
normalised = normalise_input(grayimg,shift_for_plotting=True)
plt.figure(figsize=(1,1))
print(signnames[y_train[index]])
f,((ax1,ax2,ax3),(ax4,ax5,ax6),(ax7,ax8,ax9)) = plt.subplots(3,3,)
f.tight_layout()
ax1.imshow(image)
ax1.set_title("Unprocessed image")
ax2.imshow(grayimg,cmap='gray')
ax2.set_title("Gray image")
ax3.imshow(scaled,cmap='gray')
ax3.set_title("Scaled image")
ax4.imshow(translated,cmap='gray')
ax4.set_title("Translated image")
ax5.imshow(rotated,cmap='gray')
ax5.set_title("Rotated image")
ax6.imshow(sheared,cmap='gray')
ax6.set_title("Sheared image")
ax7.imshow(cropped,cmap='gray')
ax7.set_title("Cropped image")
ax8.imshow(clahe,cmap='gray')
ax8.set_title("CLAHE filter")
ax9.imshow(normalised,cmap='gray')
ax9.set_title("Normalised image")
plt.show()
f.savefig('writeup_images/Preprocessing_img_{}.png'.format(index),dpi=300, bbox_inches = "tight")
if(plot_writeup_images):
plot_writeup_preprocess_images()
Mean: 119.006, Standard Deviation: 29.175 New Mean: 0.565, Standard Deviation: 0.386 Wild animals crossing
<Figure size 100x100 with 0 Axes>
def writeup_clahe_images():
# choose one image that is particularly dark
index = 12358
image = X_train[index]
grayimg = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
clahe = applyCLAHE(grayimg)
plt.figure(figsize=(1,1))
print(signnames[y_train[index]])
f,((ax1,ax2,ax3)) = plt.subplots(1,3,)
f.tight_layout()
ax1.imshow(image)
ax1.set_title("Unprocessed image")
ax2.imshow(grayimg,cmap='gray')
ax2.set_title("Gray image")
ax3.imshow(clahe,cmap='gray')
ax3.set_title("CLAHE image")
f.savefig('writeup_images/histogram_equalization_image_{}.png'.format(index),dpi=300, bbox_inches = "tight")
if(plot_writeup_images):
writeup_clahe_images()
No passing
<Figure size 100x100 with 0 Axes>
def applyEqualizationAndGray(images,gray_image=False):
"""
Applies the CLAHE histogram equalization
ref: https://docs.opencv.org/master/d5/daf/tutorial_py_histogram_equalization.html
:param image: list of images
:param gray_image: flag, if true the input images are already in grayscale
:return: copy of the image after processing
"""
output = np.empty_like(images,shape=(len(images),32,32,1))
for i in range(len(images)):
#gray image and CLAHE
if(gray_image == False):
grayimg = cv.cvtColor(images[i], cv.COLOR_BGR2GRAY)
else:
grayimg = images[i]
output[i] = np.expand_dims(applyCLAHE(grayimg), axis=2)
return output
X_train_CLAHE_gray = applyEqualizationAndGray(X_train)
X_valid_CLAHE_gray = applyEqualizationAndGray(X_valid)
X_test_CLAHE_gray = applyEqualizationAndGray(X_test)
if(plot_writeup_images):
plot_samples_from_all_classes(X_train_CLAHE_gray,y_train,color='gray')
plot_samples_from_all_classes(X_valid_CLAHE_gray,y_valid,color='gray')
plot_samples_from_all_classes(X_test_CLAHE_gray,y_test,color='gray')
Class 0: Speed limit (20km/h) 180 samples
-------------------------------------------------------------------------------------- Class 1: Speed limit (30km/h) 1980 samples
-------------------------------------------------------------------------------------- Class 2: Speed limit (50km/h) 2010 samples
-------------------------------------------------------------------------------------- Class 3: Speed limit (60km/h) 1260 samples
-------------------------------------------------------------------------------------- Class 4: Speed limit (70km/h) 1770 samples
-------------------------------------------------------------------------------------- Class 5: Speed limit (80km/h) 1650 samples
-------------------------------------------------------------------------------------- Class 6: End of speed limit (80km/h) 360 samples
-------------------------------------------------------------------------------------- Class 7: Speed limit (100km/h) 1290 samples
-------------------------------------------------------------------------------------- Class 8: Speed limit (120km/h) 1260 samples
-------------------------------------------------------------------------------------- Class 9: No passing 1320 samples
-------------------------------------------------------------------------------------- Class 10: No passing for vehicles over 3.5 metric tons 1800 samples
-------------------------------------------------------------------------------------- Class 11: Right-of-way at the next intersection 1170 samples
-------------------------------------------------------------------------------------- Class 12: Priority road 1890 samples
-------------------------------------------------------------------------------------- Class 13: Yield 1920 samples
-------------------------------------------------------------------------------------- Class 14: Stop 690 samples
-------------------------------------------------------------------------------------- Class 15: No vehicles 540 samples
-------------------------------------------------------------------------------------- Class 16: Vehicles over 3.5 metric tons prohibited 360 samples
-------------------------------------------------------------------------------------- Class 17: No entry 990 samples
-------------------------------------------------------------------------------------- Class 18: General caution 1080 samples
-------------------------------------------------------------------------------------- Class 19: Dangerous curve to the left 180 samples
-------------------------------------------------------------------------------------- Class 20: Dangerous curve to the right 300 samples
-------------------------------------------------------------------------------------- Class 21: Double curve 270 samples
-------------------------------------------------------------------------------------- Class 22: Bumpy road 330 samples
-------------------------------------------------------------------------------------- Class 23: Slippery road 450 samples
-------------------------------------------------------------------------------------- Class 24: Road narrows on the right 240 samples
-------------------------------------------------------------------------------------- Class 25: Road work 1350 samples
-------------------------------------------------------------------------------------- Class 26: Traffic signals 540 samples
-------------------------------------------------------------------------------------- Class 27: Pedestrians 210 samples
-------------------------------------------------------------------------------------- Class 28: Children crossing 480 samples
-------------------------------------------------------------------------------------- Class 29: Bicycles crossing 240 samples
-------------------------------------------------------------------------------------- Class 30: Beware of ice/snow 390 samples
-------------------------------------------------------------------------------------- Class 31: Wild animals crossing 690 samples
-------------------------------------------------------------------------------------- Class 32: End of all speed and passing limits 210 samples
-------------------------------------------------------------------------------------- Class 33: Turn right ahead 599 samples
-------------------------------------------------------------------------------------- Class 34: Turn left ahead 360 samples
-------------------------------------------------------------------------------------- Class 35: Ahead only 1080 samples
-------------------------------------------------------------------------------------- Class 36: Go straight or right 330 samples
-------------------------------------------------------------------------------------- Class 37: Go straight or left 180 samples
-------------------------------------------------------------------------------------- Class 38: Keep right 1860 samples
-------------------------------------------------------------------------------------- Class 39: Keep left 270 samples
-------------------------------------------------------------------------------------- Class 40: Roundabout mandatory 300 samples
-------------------------------------------------------------------------------------- Class 41: End of no passing 210 samples
-------------------------------------------------------------------------------------- Class 42: End of no passing by vehicles over 3.5 metric tons 210 samples
-------------------------------------------------------------------------------------- Class 0: Speed limit (20km/h) 30 samples
-------------------------------------------------------------------------------------- Class 1: Speed limit (30km/h) 240 samples
-------------------------------------------------------------------------------------- Class 2: Speed limit (50km/h) 240 samples
-------------------------------------------------------------------------------------- Class 3: Speed limit (60km/h) 150 samples
-------------------------------------------------------------------------------------- Class 4: Speed limit (70km/h) 210 samples
-------------------------------------------------------------------------------------- Class 5: Speed limit (80km/h) 210 samples
-------------------------------------------------------------------------------------- Class 6: End of speed limit (80km/h) 60 samples
-------------------------------------------------------------------------------------- Class 7: Speed limit (100km/h) 150 samples
-------------------------------------------------------------------------------------- Class 8: Speed limit (120km/h) 150 samples
-------------------------------------------------------------------------------------- Class 9: No passing 150 samples
-------------------------------------------------------------------------------------- Class 10: No passing for vehicles over 3.5 metric tons 210 samples
-------------------------------------------------------------------------------------- Class 11: Right-of-way at the next intersection 150 samples
-------------------------------------------------------------------------------------- Class 12: Priority road 210 samples
-------------------------------------------------------------------------------------- Class 13: Yield 240 samples
-------------------------------------------------------------------------------------- Class 14: Stop 90 samples
-------------------------------------------------------------------------------------- Class 15: No vehicles 90 samples
-------------------------------------------------------------------------------------- Class 16: Vehicles over 3.5 metric tons prohibited 60 samples
-------------------------------------------------------------------------------------- Class 17: No entry 120 samples
-------------------------------------------------------------------------------------- Class 18: General caution 120 samples
-------------------------------------------------------------------------------------- Class 19: Dangerous curve to the left 30 samples
-------------------------------------------------------------------------------------- Class 20: Dangerous curve to the right 60 samples
-------------------------------------------------------------------------------------- Class 21: Double curve 60 samples
-------------------------------------------------------------------------------------- Class 22: Bumpy road 60 samples
-------------------------------------------------------------------------------------- Class 23: Slippery road 60 samples
-------------------------------------------------------------------------------------- Class 24: Road narrows on the right 30 samples
-------------------------------------------------------------------------------------- Class 25: Road work 150 samples
-------------------------------------------------------------------------------------- Class 26: Traffic signals 60 samples
-------------------------------------------------------------------------------------- Class 27: Pedestrians 30 samples
-------------------------------------------------------------------------------------- Class 28: Children crossing 60 samples
-------------------------------------------------------------------------------------- Class 29: Bicycles crossing 30 samples
-------------------------------------------------------------------------------------- Class 30: Beware of ice/snow 60 samples
-------------------------------------------------------------------------------------- Class 31: Wild animals crossing 90 samples
-------------------------------------------------------------------------------------- Class 32: End of all speed and passing limits 30 samples
-------------------------------------------------------------------------------------- Class 33: Turn right ahead 90 samples
-------------------------------------------------------------------------------------- Class 34: Turn left ahead 60 samples
-------------------------------------------------------------------------------------- Class 35: Ahead only 120 samples
-------------------------------------------------------------------------------------- Class 36: Go straight or right 60 samples
-------------------------------------------------------------------------------------- Class 37: Go straight or left 30 samples
-------------------------------------------------------------------------------------- Class 38: Keep right 210 samples
-------------------------------------------------------------------------------------- Class 39: Keep left 30 samples
-------------------------------------------------------------------------------------- Class 40: Roundabout mandatory 60 samples
-------------------------------------------------------------------------------------- Class 41: End of no passing 30 samples
-------------------------------------------------------------------------------------- Class 42: End of no passing by vehicles over 3.5 metric tons 30 samples
-------------------------------------------------------------------------------------- Class 0: Speed limit (20km/h) 60 samples
-------------------------------------------------------------------------------------- Class 1: Speed limit (30km/h) 720 samples
-------------------------------------------------------------------------------------- Class 2: Speed limit (50km/h) 750 samples
-------------------------------------------------------------------------------------- Class 3: Speed limit (60km/h) 450 samples
-------------------------------------------------------------------------------------- Class 4: Speed limit (70km/h) 660 samples
-------------------------------------------------------------------------------------- Class 5: Speed limit (80km/h) 630 samples
-------------------------------------------------------------------------------------- Class 6: End of speed limit (80km/h) 150 samples
-------------------------------------------------------------------------------------- Class 7: Speed limit (100km/h) 450 samples
-------------------------------------------------------------------------------------- Class 8: Speed limit (120km/h) 450 samples
-------------------------------------------------------------------------------------- Class 9: No passing 480 samples
-------------------------------------------------------------------------------------- Class 10: No passing for vehicles over 3.5 metric tons 660 samples
-------------------------------------------------------------------------------------- Class 11: Right-of-way at the next intersection 420 samples
-------------------------------------------------------------------------------------- Class 12: Priority road 690 samples
-------------------------------------------------------------------------------------- Class 13: Yield 720 samples
-------------------------------------------------------------------------------------- Class 14: Stop 270 samples
-------------------------------------------------------------------------------------- Class 15: No vehicles 210 samples
-------------------------------------------------------------------------------------- Class 16: Vehicles over 3.5 metric tons prohibited 150 samples
-------------------------------------------------------------------------------------- Class 17: No entry 360 samples
-------------------------------------------------------------------------------------- Class 18: General caution 390 samples
-------------------------------------------------------------------------------------- Class 19: Dangerous curve to the left 60 samples
-------------------------------------------------------------------------------------- Class 20: Dangerous curve to the right 90 samples
-------------------------------------------------------------------------------------- Class 21: Double curve 90 samples
-------------------------------------------------------------------------------------- Class 22: Bumpy road 120 samples
-------------------------------------------------------------------------------------- Class 23: Slippery road 150 samples
-------------------------------------------------------------------------------------- Class 24: Road narrows on the right 90 samples
-------------------------------------------------------------------------------------- Class 25: Road work 480 samples
-------------------------------------------------------------------------------------- Class 26: Traffic signals 180 samples
-------------------------------------------------------------------------------------- Class 27: Pedestrians 60 samples
-------------------------------------------------------------------------------------- Class 28: Children crossing 150 samples
-------------------------------------------------------------------------------------- Class 29: Bicycles crossing 90 samples
-------------------------------------------------------------------------------------- Class 30: Beware of ice/snow 150 samples
-------------------------------------------------------------------------------------- Class 31: Wild animals crossing 270 samples
-------------------------------------------------------------------------------------- Class 32: End of all speed and passing limits 60 samples
-------------------------------------------------------------------------------------- Class 33: Turn right ahead 210 samples
-------------------------------------------------------------------------------------- Class 34: Turn left ahead 120 samples
-------------------------------------------------------------------------------------- Class 35: Ahead only 390 samples
-------------------------------------------------------------------------------------- Class 36: Go straight or right 120 samples
-------------------------------------------------------------------------------------- Class 37: Go straight or left 60 samples
-------------------------------------------------------------------------------------- Class 38: Keep right 690 samples
-------------------------------------------------------------------------------------- Class 39: Keep left 90 samples
-------------------------------------------------------------------------------------- Class 40: Roundabout mandatory 90 samples
-------------------------------------------------------------------------------------- Class 41: End of no passing 60 samples
-------------------------------------------------------------------------------------- Class 42: End of no passing by vehicles over 3.5 metric tons 90 samples
--------------------------------------------------------------------------------------
def save_pickle_file(filename,features,labels):
"""
Helper function to save features and lables as
a dict inside a pickle file
:param filename: name of the pickle file
:param features: features to save
:param labels: labels to save
:return: nothing
"""
assert(len(features) == len(labels))
data = {'features':features,'labels':labels}
with open(filename, mode='wb') as f:
pickle.dump(data, f)
# Sometimes(0.5, ...) applies the given augmenter in 50% of
# all cases, e.g. Sometimes(0.5, GaussianBlur(0.3))
# would blur roughly every second image.
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
#the sequence of filters to apply to the images
seq = iaa.Sequential(
[
iaa.SomeOf((1, 2),
[
# crop images by -10% to 10% of their height/width
sometimes(iaa.CropAndPad(
percent=(-0.1, 0.1),
pad_cval=(0, 255)
)),
sometimes(iaa.Affine(
# scale between 80% and 120% of the image (per axis)
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
# translate by -20 to +20 percent (per axis)
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
# rotate by -45 to +45 degrees
rotate=(-45, 45),
# shear by -20 to +20 degrees
shear=(-20, 20),
)),
],
random_order=True
)
],
random_order=True
)
def populate_class_size_normalisation_table(item_counts):
"""
Helper function that returns a dataframe with the info
about which image class needs to augmented and how many
fake images need to be generated
:param item_counts: number of images per each class
:return: dataframe
"""
#save the size of the largest class
max_val = np.max(item_counts)
#create the dataframe
aug_table = pd.DataFrame({'class_id':[i for i in range(43)],
'img_cnt':item_counts,
'should_be':max_val,
'to_add':0,
'class_aug_factor':0})
#populate the dataframe by scanning the
#input vector
for current_class, val in enumerate(item_counts):
add = max_val - val
aug_factor = int(np.floor(add/val))+1
aug_table.at[current_class,'to_add']=add
aug_table.at[current_class,'class_aug_factor']=aug_factor
return aug_table
def augment(aug_table,images,labels,single_channel_img=True,augmentations=None,augment_percent=50):
"""
Function that generates fake images. If augment_percent==0
the function generates data so that all classes have the same
number of images as the largest class of the dataset
:param aug_table: dataframe that indicates how many
fake images per class must be generated
:param images: original images
:param labels: corresponding labels
:param augmentations: imaug filters to apply for fake
image generation
:param augment_percent:
0 means generate image to balance all classes
100 means balance all classes and double each class size
:return: new_images,new_lables
"""
new_images = np.array([])
new_lables = np.array([])
for row in range(aug_table.shape[0]):
#current class
current_class = aug_table.loc[row].at['class_id']
# get the indexes of the images whose label
# corresponds to current_class
itemindex = np.where(labels==current_class)
# select all images that correspond to the current class
images_of_this_class = images[itemindex[0:2]]
#current class population?
class_size = aug_table.loc[row].at['img_cnt']
assert(class_size == len(images_of_this_class))
#target size for this class
target_size = aug_table.loc[row].at['should_be']
if (augment_percent>0):
target_size = target_size + np.floor(target_size*augment_percent/100)
print("augmenting class {} from {} to {} unique images".format(current_class,class_size,target_size))
#until the target size is not reached generate new
#fake images. If an image already exists, discard it
#and generate a new onw
while (len(images_of_this_class)<target_size):
#pick one of the original images
random_img_idx = np.random.randint(0,class_size)
#pick the image
original = images_of_this_class[random_img_idx]
#pass it to some filters
edited_img = augmentations(image=original.squeeze())
if(single_channel_img):
edited_img = np.expand_dims(edited_img, axis=2)
#check if edited image already exists, if not,
#add it to the array
for i in range(len(images_of_this_class)):
if np.array_equal(edited_img, images_of_this_class[i]):
#print("image already exist")
break
#add the image and the label
edited_img = np.expand_dims(edited_img, axis=0)
images_of_this_class = np.append(images_of_this_class,edited_img,axis=0)
labels_of_this_class = np.array([current_class]*len(images_of_this_class))
assert(len(labels_of_this_class) == len(images_of_this_class))
#add new class images to the dataset
if (new_images.shape[0]==0):
new_images = images_of_this_class
new_lables = labels_of_this_class
else:
new_images= np.append(new_images,images_of_this_class,axis=0)
new_lables= np.append(new_lables,labels_of_this_class,axis=0)
return new_images,new_lables
aug_table = None
df = pd.DataFrame({'sign':list(y_train)})
item_counts = df["sign"].value_counts().sort_index()
aug_table = populate_class_size_normalisation_table(item_counts)
#0=only generate data to balance the datasets so
# that all classes have the same size
#100= generate data to balance the datasets AND
# increases the all classes by 100% (double the size)
fake_data_augmentation_factor = 100
if(generate_fake_data_for_augmentation):
x_augmented,y_augmented = augment(
aug_table,
X_train,
y_train,
single_channel_img=False,
augmentations=seq,
augment_percent=fake_data_augmentation_factor)
if(plot_writeup_images):
plot_samples_from_all_classes(x_augmented,y_augmented)
#visualizing and saving the new training data distribution
plot_sign_count_bar(signnames,y_augmented,dataset_name='Training_balanced_aug_factor{}'.format(fake_data_augmentation_factor))
augmenting class 0 from 180 to 4020.0 unique images augmenting class 1 from 1980 to 4020.0 unique images augmenting class 2 from 2010 to 4020.0 unique images augmenting class 3 from 1260 to 4020.0 unique images augmenting class 4 from 1770 to 4020.0 unique images augmenting class 5 from 1650 to 4020.0 unique images augmenting class 6 from 360 to 4020.0 unique images augmenting class 7 from 1290 to 4020.0 unique images augmenting class 8 from 1260 to 4020.0 unique images augmenting class 9 from 1320 to 4020.0 unique images augmenting class 10 from 1800 to 4020.0 unique images augmenting class 11 from 1170 to 4020.0 unique images augmenting class 12 from 1890 to 4020.0 unique images augmenting class 13 from 1920 to 4020.0 unique images augmenting class 14 from 690 to 4020.0 unique images augmenting class 15 from 540 to 4020.0 unique images augmenting class 16 from 360 to 4020.0 unique images augmenting class 17 from 990 to 4020.0 unique images augmenting class 18 from 1080 to 4020.0 unique images augmenting class 19 from 180 to 4020.0 unique images augmenting class 20 from 300 to 4020.0 unique images augmenting class 21 from 270 to 4020.0 unique images augmenting class 22 from 330 to 4020.0 unique images augmenting class 23 from 450 to 4020.0 unique images augmenting class 24 from 240 to 4020.0 unique images augmenting class 25 from 1350 to 4020.0 unique images augmenting class 26 from 540 to 4020.0 unique images augmenting class 27 from 210 to 4020.0 unique images augmenting class 28 from 480 to 4020.0 unique images augmenting class 29 from 240 to 4020.0 unique images augmenting class 30 from 390 to 4020.0 unique images augmenting class 31 from 690 to 4020.0 unique images augmenting class 32 from 210 to 4020.0 unique images augmenting class 33 from 599 to 4020.0 unique images augmenting class 34 from 360 to 4020.0 unique images augmenting class 35 from 1080 to 4020.0 unique images augmenting class 36 from 330 to 4020.0 unique images augmenting class 37 from 180 to 4020.0 unique images augmenting class 38 from 1860 to 4020.0 unique images augmenting class 39 from 270 to 4020.0 unique images augmenting class 40 from 300 to 4020.0 unique images augmenting class 41 from 210 to 4020.0 unique images augmenting class 42 from 210 to 4020.0 unique images Class 0: Speed limit (20km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 1: Speed limit (30km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 2: Speed limit (50km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 3: Speed limit (60km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 4: Speed limit (70km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 5: Speed limit (80km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 6: End of speed limit (80km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 7: Speed limit (100km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 8: Speed limit (120km/h) 4020 samples
-------------------------------------------------------------------------------------- Class 9: No passing 4020 samples
-------------------------------------------------------------------------------------- Class 10: No passing for vehicles over 3.5 metric tons 4020 samples
-------------------------------------------------------------------------------------- Class 11: Right-of-way at the next intersection 4020 samples
-------------------------------------------------------------------------------------- Class 12: Priority road 4020 samples
-------------------------------------------------------------------------------------- Class 13: Yield 4020 samples
-------------------------------------------------------------------------------------- Class 14: Stop 4020 samples
-------------------------------------------------------------------------------------- Class 15: No vehicles 4020 samples
-------------------------------------------------------------------------------------- Class 16: Vehicles over 3.5 metric tons prohibited 4020 samples
-------------------------------------------------------------------------------------- Class 17: No entry 4020 samples
-------------------------------------------------------------------------------------- Class 18: General caution 4020 samples
-------------------------------------------------------------------------------------- Class 19: Dangerous curve to the left 4020 samples
-------------------------------------------------------------------------------------- Class 20: Dangerous curve to the right 4020 samples
-------------------------------------------------------------------------------------- Class 21: Double curve 4020 samples
-------------------------------------------------------------------------------------- Class 22: Bumpy road 4020 samples
-------------------------------------------------------------------------------------- Class 23: Slippery road 4020 samples
-------------------------------------------------------------------------------------- Class 24: Road narrows on the right 4020 samples
-------------------------------------------------------------------------------------- Class 25: Road work 4020 samples
-------------------------------------------------------------------------------------- Class 26: Traffic signals 4020 samples
-------------------------------------------------------------------------------------- Class 27: Pedestrians 4020 samples
-------------------------------------------------------------------------------------- Class 28: Children crossing 4020 samples
-------------------------------------------------------------------------------------- Class 29: Bicycles crossing 4020 samples
-------------------------------------------------------------------------------------- Class 30: Beware of ice/snow 4020 samples
-------------------------------------------------------------------------------------- Class 31: Wild animals crossing 4020 samples
-------------------------------------------------------------------------------------- Class 32: End of all speed and passing limits 4020 samples
-------------------------------------------------------------------------------------- Class 33: Turn right ahead 4020 samples
-------------------------------------------------------------------------------------- Class 34: Turn left ahead 4020 samples
-------------------------------------------------------------------------------------- Class 35: Ahead only 4020 samples
-------------------------------------------------------------------------------------- Class 36: Go straight or right 4020 samples
-------------------------------------------------------------------------------------- Class 37: Go straight or left 4020 samples
-------------------------------------------------------------------------------------- Class 38: Keep right 4020 samples
-------------------------------------------------------------------------------------- Class 39: Keep left 4020 samples
-------------------------------------------------------------------------------------- Class 40: Roundabout mandatory 4020 samples
-------------------------------------------------------------------------------------- Class 41: End of no passing 4020 samples
-------------------------------------------------------------------------------------- Class 42: End of no passing by vehicles over 3.5 metric tons 4020 samples
--------------------------------------------------------------------------------------
X_train = normalise_input(X_train_CLAHE_gray)
X_valid = normalise_input(X_valid_CLAHE_gray)
X_test = normalise_input(X_test_CLAHE_gray)
print("min:{},max:{}".format(np.min(X_train),np.max(X_train)))
print("min:{},max:{}".format(np.min(X_valid),np.max(X_valid)))
print("min:{},max:{}".format(np.min(X_test),np.max(X_test)))
if(generate_fake_data_for_augmentation):
X_train_aug_CLAHE_gray = applyEqualizationAndGray(x_augmented)
X_train_aug_norm = normalise_input(X_train_aug_CLAHE_gray)
print("min:{},max:{}".format(np.min(X_train_aug_norm),np.max(X_train_aug_norm)))
Mean: 132.128, Standard Deviation: 62.166 New Mean: -0.031, Standard Deviation: 0.759 Mean: 132.219, Standard Deviation: 62.615 New Mean: -0.033, Standard Deviation: 0.760 Mean: 132.785, Standard Deviation: 63.019 New Mean: -0.034, Standard Deviation: 0.761 min:-1.0,max:1.0 min:-1.0,max:1.0 min:-1.0,max:1.0 Mean: 127.834, Standard Deviation: 63.112 New Mean: -0.037, Standard Deviation: 0.766 min:-1.0,max:1.0
# save new pickle files, training dataset,
# augmented training dataset, validation dataset
# and testing dataset
filename= '../traffic-signs-data/train_clahe_norm_no_aug.p'
save_pickle_file(filename,X_train,y_train)
if(generate_fake_data_for_augmentation):
filename = '../traffic-signs-data/train_CLAHE_norm_balanced_aug_factor_{}.p'.format(fake_data_augmentation_factor)
save_pickle_file(filename,X_train_aug_norm,y_augmented)
filename= '../traffic-signs-data/valid_clahe_norm_no_aug.p'
save_pickle_file(filename,X_valid,y_valid)
filename= '../traffic-signs-data/test_clahe_norm_no_aug.p'
save_pickle_file(filename,X_test,y_test)
training_file= '../traffic-signs-data/train_clahe_norm_no_aug.p'
training_file_aug_0= '../traffic-signs-data/train_CLAHE_norm_balanced_aug_factor_0.p'
training_file_aug_100= '../traffic-signs-data/train_CLAHE_norm_balanced_aug_factor_100.p'
validation_file= '../traffic-signs-data/valid_clahe_norm_no_aug.p'
testing_file= '../traffic-signs-data/test_clahe_norm_no_aug.p'
with open(training_file, mode='rb') as f:
train = pickle.load(f)
with open(training_file_aug_0, mode='rb') as f:
train_aug_0 = pickle.load(f)
with open(training_file_aug_100, mode='rb') as f:
train_aug_100 = pickle.load(f)
with open(validation_file, mode='rb') as f:
valid = pickle.load(f)
with open(testing_file, mode='rb') as f:
test = pickle.load(f)
X_train, y_train = train['features'], train['labels']
X_train_aug_0, y_train_aug_0 = train_aug_0['features'], train_aug_0['labels']
X_train_aug_100, y_train_aug_100 = train_aug_100['features'], train_aug_100['labels']
X_valid, y_valid = valid['features'], valid['labels']
X_test, y_test = test['features'], test['labels']
def get_n_classes(labels):
return len(set(labels))
print_stats(X_train,y_train,'original training')
print_stats(X_train_aug_0,y_train_aug_0,'augmented training')
print_stats(X_train_aug_100,y_train_aug_100,'augmented training')
print_stats(X_valid,y_valid,'validation')
print_stats(X_test,y_test,'testing')
Stats for the original training dataset --------------------- Number of examples = 34799 Image data shape = (32, 32, 1) Number of classes = 43 --------------------- Stats for the augmented training dataset --------------------- Number of examples = 86430 Image data shape = (32, 32, 1) Number of classes = 43 --------------------- Stats for the augmented training dataset --------------------- Number of examples = 172860 Image data shape = (32, 32, 1) Number of classes = 43 --------------------- Stats for the validation dataset --------------------- Number of examples = 4410 Image data shape = (32, 32, 1) Number of classes = 43 --------------------- Stats for the testing dataset --------------------- Number of examples = 12630 Image data shape = (32, 32, 1) Number of classes = 43 ---------------------
# Convert class vectors to binary class matrices.
n_classes = get_n_classes(y_train)
y_train = to_categorical(y_train, n_classes)
y_train_aug_0 = to_categorical(y_train_aug_0, n_classes)
y_train_aug_100 = to_categorical(y_train_aug_100, n_classes)
y_valid = to_categorical(y_valid, n_classes)
y_test = to_categorical(y_test, n_classes)
assert(len(X_train) == len(y_train))
assert(len(X_train_aug_0) == len(y_train_aug_0))
assert(len(X_train_aug_100) == len(y_train_aug_100))
assert(len(X_valid) == len(y_valid))
assert(len(X_test) == len(y_test))
#shuffle data
X_train, y_train = shuffle(X_train, y_train)
X_train_aug_0, y_train_aug_0 = shuffle(X_train_aug_0, y_train_aug_0)
X_train_aug_100, y_train_aug_100 = shuffle(X_train_aug_100, y_train_aug_100)
#uncomment the dataset and label to use for training
#training_dataset_features = X_train
#training_dataset_labels = y_train
#training_dataset_features = X_train_aug_0
#training_dataset_labels = y_train_aug_0
training_dataset_features = X_train_aug_100
training_dataset_labels = y_train_aug_100
input_shape = training_dataset_features[0].shape
#dropout parameter for layer 4
p4 = .3
#dropout parameter for layer 5
p5 = .2
#kernel size of all conv 2D layers (layer 1,2 and 3)
conv_kern = 3
#depth of the conv 2D layers (layer 1,2 and 3)
conv_depths = [32,64,128]
#num epochs
training_epochs=200
#use early stop condition
early_stop_active = True
#training batch size
training_batch_size = 512
print_stats(training_dataset_features,training_dataset_labels,'selected training')
Stats for the selected training dataset --------------------- Number of examples = 172860 Image data shape = (32, 32, 1) Number of classes = 43 ---------------------
# Modified LeNet-5 model with three convolutional layers
model = Sequential(name='c1{}_c2{}_c3{}_k{}_p4{}_p5{}_es{}'.format(
conv_depths[0],
conv_depths[1],
conv_depths[2],
conv_kern,
p4,
p5,
early_stop_active))
#Layer 1: Convolutional. Input = 32x32x1. Output = 30x30x32.
model.add(
Conv2D(
conv_depths[0],
kernel_size=(conv_kern, conv_kern),
strides=(1, 1),
activation='relu',
input_shape=input_shape,
padding="valid",
name='layer1'))
#Pooling. Input = 30x30x32. Output = 15x15x32.
model.add(
MaxPooling2D(
pool_size=(2, 2),
padding='same',
name='maxPool1'))
#Layer 2: Convolutional. Output = 13x13x64.
model.add(
Conv2D(
conv_depths[1],
kernel_size=(conv_kern, conv_kern),
strides=(1, 1),
activation='relu',
padding='valid',
name='layer2'))
#Pooling. Input = 13x13x64. Output = 7x7x64.
model.add(
MaxPooling2D(
pool_size=(2, 2),
padding='same',
name='maxPool2'))
#Layer 3: Convolutional. Output = 5x5x128.
model.add(
Conv2D(
conv_depths[2],
kernel_size=(conv_kern, conv_kern),
strides=(1, 1),
activation='relu',
padding='valid',
name='layer3'))
#Pooling. Input = 5x5x128. Output = 3x3x128.
model.add(
MaxPooling2D(
pool_size=(2, 2),
padding='same',
name='maxPool3'))
#Flatten. Input = 3x3x128. Output = 1152.
model.add(Flatten(name='flatten'))
#Layer 4: Fully Connected. Input = 1152. Output = 120.
model.add(Dense(120, activation='relu',name='layer4'))
#Dropout
if(p4<1.0):
model.add(Dropout(p4,name='dropout4'))
#Layer 5: Fully Connected. Input = 120. Output = 84.
model.add(Dense(84, activation='relu',name='layer5'))
#Dropout
if(p5<1.0):
model.add(Dropout(p5,name='dropout5'))
#Layer 5: Fully Connected. Input = 84. Output = n_classes.
model.add(Dense(n_classes, activation='softmax',name='layer5-out'))
model.summary()
#Optimizer choice
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
#Compile the model
model.compile(optimizer=opt,
loss=categorical_crossentropy,
metrics=['accuracy'])
Model: "c132_c264_c3128_k3_p40.3_p50.2_esTrue" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= layer1 (Conv2D) (None, 30, 30, 32) 320 _________________________________________________________________ maxPool1 (MaxPooling2D) (None, 15, 15, 32) 0 _________________________________________________________________ layer2 (Conv2D) (None, 13, 13, 64) 18496 _________________________________________________________________ maxPool2 (MaxPooling2D) (None, 7, 7, 64) 0 _________________________________________________________________ layer3 (Conv2D) (None, 5, 5, 128) 73856 _________________________________________________________________ maxPool3 (MaxPooling2D) (None, 3, 3, 128) 0 _________________________________________________________________ flatten (Flatten) (None, 1152) 0 _________________________________________________________________ layer4 (Dense) (None, 120) 138360 _________________________________________________________________ dropout4 (Dropout) (None, 120) 0 _________________________________________________________________ layer5 (Dense) (None, 84) 10164 _________________________________________________________________ dropout5 (Dropout) (None, 84) 0 _________________________________________________________________ layer5-out (Dense) (None, 43) 3655 ================================================================= Total params: 244,851 Trainable params: 244,851 Non-trainable params: 0 _________________________________________________________________
if(train_model_flag):
# Place the logs in a timestamped subdirectory
# This allows to easy select different training runs
# In order not to overwrite some data, it is useful to have a name with a timestamp
time_stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
complete_model_name = '{}-{}'.format(model.name, time_stamp)
log_dir="logs/fit/" +complete_model_name
# Specify the callback object
# callback for tensorboard
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
# early stop callback
es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)
if(train_model_flag):
if(early_stop_active):
active_callbacks = [tensorboard_callback,es_callback]
else:
active_callbacks = [tensorboard_callback]
history = model.fit(x=training_dataset_features,
y=training_dataset_labels,
epochs=training_epochs,
batch_size = training_batch_size,
validation_data=(X_valid, y_valid),
callbacks=active_callbacks,
verbose=1)
tf.keras.models.save_model(model,'./models/{}'.format(complete_model_name))
results = model.evaluate(X_test,
y_test,
batch_size=128,
verbose=0)
print("test loss, test acc:", results)
history.history
Epoch 1/200 338/338 [==============================] - 151s 445ms/step - loss: 2.6315 - accuracy: 0.2795 - val_loss: 0.3925 - val_accuracy: 0.8995 Epoch 2/200 338/338 [==============================] - 142s 419ms/step - loss: 0.9771 - accuracy: 0.7114 - val_loss: 0.1843 - val_accuracy: 0.9537 Epoch 3/200 338/338 [==============================] - 138s 408ms/step - loss: 0.6467 - accuracy: 0.8020 - val_loss: 0.1087 - val_accuracy: 0.9705 Epoch 4/200 338/338 [==============================] - 137s 404ms/step - loss: 0.4814 - accuracy: 0.8504 - val_loss: 0.0914 - val_accuracy: 0.9751 Epoch 5/200 338/338 [==============================] - 136s 401ms/step - loss: 0.3760 - accuracy: 0.8824 - val_loss: 0.0779 - val_accuracy: 0.9773 Epoch 6/200 338/338 [==============================] - 143s 424ms/step - loss: 0.3096 - accuracy: 0.9025 - val_loss: 0.0884 - val_accuracy: 0.9794 Epoch 7/200 338/338 [==============================] - 143s 422ms/step - loss: 0.2538 - accuracy: 0.9196 - val_loss: 0.0591 - val_accuracy: 0.9834 Epoch 8/200 338/338 [==============================] - 146s 433ms/step - loss: 0.2147 - accuracy: 0.9314 - val_loss: 0.0853 - val_accuracy: 0.9812 Epoch 9/200 338/338 [==============================] - 139s 412ms/step - loss: 0.1895 - accuracy: 0.9389 - val_loss: 0.0702 - val_accuracy: 0.9821 Epoch 10/200 338/338 [==============================] - 140s 413ms/step - loss: 0.1654 - accuracy: 0.9466 - val_loss: 0.0969 - val_accuracy: 0.9798 Epoch 11/200 338/338 [==============================] - 142s 421ms/step - loss: 0.1491 - accuracy: 0.9515 - val_loss: 0.0982 - val_accuracy: 0.9823 Epoch 12/200 338/338 [==============================] - 148s 437ms/step - loss: 0.1306 - accuracy: 0.9572 - val_loss: 0.0942 - val_accuracy: 0.9821 Epoch 13/200 338/338 [==============================] - 141s 417ms/step - loss: 0.1180 - accuracy: 0.9615 - val_loss: 0.1219 - val_accuracy: 0.9832 Epoch 14/200 338/338 [==============================] - 145s 427ms/step - loss: 0.1087 - accuracy: 0.9647 - val_loss: 0.0843 - val_accuracy: 0.9834 Epoch 15/200 338/338 [==============================] - 138s 409ms/step - loss: 0.0977 - accuracy: 0.9681 - val_loss: 0.1587 - val_accuracy: 0.9823 Epoch 16/200 338/338 [==============================] - 129s 381ms/step - loss: 0.0872 - accuracy: 0.9714 - val_loss: 0.1176 - val_accuracy: 0.9828 Epoch 17/200 338/338 [==============================] - 128s 378ms/step - loss: 0.0841 - accuracy: 0.9722 - val_loss: 0.1878 - val_accuracy: 0.9812 INFO:tensorflow:Assets written to: ./models/c132_c264_c3128_k3_p40.3_p50.2_esTrue-20210516-123943/assets test loss, test acc: [0.21022966504096985, 0.9569279551506042]
%load_ext tensorboard
%tensorboard --logdir logs/fit
Reusing TensorBoard on port 6007 (pid 35351), started 3 days, 0:12:26 ago. (Use '!kill 35351' to kill it.)
To give yourself more insight into how your model is working, download at least five pictures of German traffic signs from the web and use your model to predict the traffic sign type.
You may find signnames.csv useful as it contains mappings from the class id (integer) to the actual sign name.
def resizeAndPad(pil_img,target_h,target_w):
"""
Helper function to resize and pad sign pictures I took
:param pil_img: pil image file
:param target_h: target height
:param target_w: target width
:return: processed image
"""
pil_img.thumbnail((target_h,target_w))
width, height = pil_img.size
if width == height:
return pil_img
elif width > height:
result = Image.new(pil_img.mode, (width, width))
result.paste(pil_img, (0, (width - height) // 2))
return result
else:
result = Image.new(pil_img.mode, (height, height))
result.paste(pil_img, ((height - width) // 2, 0))
return result
def preprocess_own_pictures():
"""
Helper function to process pictures I took and convert them
in the right format. The function returns a dict similar to
the original traffic sign dataset
:return: a dict for features and labels
"""
#make sure to be in the right directory
os.chdir(notebook_root_path)
os.chdir("./additional-signs")
cwd = os.getcwd()
#prepare the empty dict, keep the color version of the image
#for visualization purposes
data = {'features':np.empty([0,32,32,1]),
'features_color':np.empty([0,32,32,3]),
'labels':np.empty([0])}
for file in glob.glob("*.png"):
#process color image and add it to the dict
color = Image.open(file).convert('RGB')
im_new = resizeAndPad(color,32,32)
pix = np.array(im_new)
pix_exd = np.expand_dims(pix,axis=0)
#add to the dict
data['features_color'] = np.append(data['features_color'],pix_exd,axis=0)
#process gray image nd add it to the dict
gray = Image.open(file).convert('L')
im_new = resizeAndPad(gray,32,32)
pix = np.array(im_new)
pix_exd = np.expand_dims(np.expand_dims(pix,axis=2),axis=0)
#add to the dict
data['features'] = np.append(data['features'],pix_exd,axis=0)
#add label to the dict
sign_number = file.split('_')[1]
data['labels'] = np.append(data['labels'],int(sign_number))
os.chdir("../")
return data
def writeup_single_image(color,gray,norm,label,img_num):
"""
Helper function to plot writeup images and save them
:param color: color image
:param gray: gray (CLAHE) image
:param norm: normalised image
:param label: image label
:param img_num: image number to use as filename
:return: save an image
"""
plt.figure(figsize=(1,1))
f,((ax1,ax2,ax3)) = plt.subplots(1,3,)
f.tight_layout()
ax1.imshow(color)
ax1.set_title("Unprocessed image")
ax2.imshow(gray,cmap='gray')
ax2.set_title("CLAHE image")
ax3.imshow(norm,cmap='gray')
ax3.set_title("Normalised image")
f.suptitle(label, fontsize=16)
f.savefig('writeup_images/Own_image_{}.png'.format(img_num),
dpi=300,
bbox_inches = "tight")
if (process_own_images):
own_pics = preprocess_own_pictures()
X_own_test_equal = applyEqualizationAndGray(
np.uint8(own_pics['features_color']),
gray_image=False)
X_own_test_norm = normalise_input(X_own_test_equal)
print("min:{},max:{}".format(np.min(X_own_test_norm),np.max(X_own_test_norm)))
assert(len(X_own_test_norm) == len(own_pics['labels']))
os.chdir(notebook_root_path)
filename= '../traffic-signs-data/own_pictures_gray_norm.p'
save_pickle_file(filename,X_own_test_norm,own_pics['labels'])
Mean: 134.888, Standard Deviation: 64.208 New Mean: -0.018, Standard Deviation: 0.762 min:-1.0,max:1.0
own_testing_file = '../traffic-signs-data/own_pictures_gray_norm.p'
with open(own_testing_file, mode='rb') as f:
own_test = pickle.load(f)
X_own_test,y_own_test = own_test['features'], own_test['labels']
y_own_test = to_categorical(y_own_test, n_classes)
assert(len(X_own_test) == len(y_own_test))
n_own_test = len(y_own_test)
print_stats(X_own_test,y_own_test,"own images")
Stats for the own images dataset --------------------- Number of examples = 54 Image data shape = (32, 32, 1) Number of classes = 43 ---------------------
if(plot_writeup_images):
plot_samples_from_all_classes(X_own_test,y_own_test,color='gray')
plot_sign_count_bar(signnames,y_own_test,dataset_name='Own')
#pick 5 unique pictures out the dataset
selection_own_images = [0,1,2,7,9]
for img_index in selection_own_images:
color = (own_pics['features_color'][img_index]).astype(np.uint8)
gray = X_own_test_equal[img_index]
norm = X_own_test_norm[img_index]
label = signnames[int(own_pics['labels'][img_index])]
writeup_single_image(color,gray,norm,label,img_index)
Class 9: No passing 8 samples
-------------------------------------------------------------------------------------- Class 12: Priority road 9 samples
-------------------------------------------------------------------------------------- Class 13: Yield 3 samples
-------------------------------------------------------------------------------------- Class 17: No entry 6 samples
-------------------------------------------------------------------------------------- Class 26: Traffic signals 1 samples
-------------------------------------------------------------------------------------- Class 33: Turn right ahead 1 samples
-------------------------------------------------------------------------------------- Class 34: Turn left ahead 5 samples
-------------------------------------------------------------------------------------- Class 35: Ahead only 10 samples
-------------------------------------------------------------------------------------- Class 36: Go straight or right 5 samples
-------------------------------------------------------------------------------------- Class 38: Keep right 3 samples
-------------------------------------------------------------------------------------- Class 40: Roundabout mandatory 3 samples
--------------------------------------------------------------------------------------
<Figure size 100x100 with 0 Axes>
<Figure size 100x100 with 0 Axes>
<Figure size 100x100 with 0 Axes>
<Figure size 100x100 with 0 Axes>
<Figure size 100x100 with 0 Axes>
#model saved with early stop conditions
model_original_es = 'c132_c264_c3128_k3_p40.3_p50.2_esTrue-20210515-113756'
model_balanced_es = 'c132_c264_c3128_k3_p40.3_p50.2_esTrue-20210515-115926'
model_balanced_and_augmented_100_es = 'c132_c264_c3128_k3_p40.3_p50.2_esTrue-20210515-131619'
#choose the model to load and use for eval
model_file = model_balanced_and_augmented_100_es
loaded_model = tf.keras.models.load_model('./models/{}'.format(model_file))
loaded_model.summary()
Model: "c132_c264_c3128_k3_p40.3_p50.2_esTrue" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= layer1 (Conv2D) (None, 30, 30, 32) 320 _________________________________________________________________ maxPool1 (MaxPooling2D) (None, 15, 15, 32) 0 _________________________________________________________________ layer2 (Conv2D) (None, 13, 13, 64) 18496 _________________________________________________________________ maxPool2 (MaxPooling2D) (None, 7, 7, 64) 0 _________________________________________________________________ layer3 (Conv2D) (None, 5, 5, 128) 73856 _________________________________________________________________ maxPool3 (MaxPooling2D) (None, 3, 3, 128) 0 _________________________________________________________________ flatten (Flatten) (None, 1152) 0 _________________________________________________________________ layer4 (Dense) (None, 120) 138360 _________________________________________________________________ dropout4 (Dropout) (None, 120) 0 _________________________________________________________________ layer5 (Dense) (None, 84) 10164 _________________________________________________________________ dropout5 (Dropout) (None, 84) 0 _________________________________________________________________ layer5-out (Dense) (None, 43) 3655 ================================================================= Total params: 244,851 Trainable params: 244,851 Non-trainable params: 0 _________________________________________________________________
def pred_details(predictions,features,labels,signnames):
"""
build an array where each element is a dictionary
that contains details about the prediction
:param predictions: predictions matrix size (dataset size x predictions)
:param features: dataset features
:param labels: dataset labels
:param signnames: string names of the traffic signs
:return: array of dict
"""
results = []
for i in range(len(predictions)):
#get the position of the top 5 arguments
top5 = predictions[i].argsort()[-5:][::-1]
pred={'name':signnames[top5[0]],
'number':top5[0],
'certainty':predictions[i][top5[0]],
'image':features[i].squeeze(),
'correct_class':signnames[np.argmax(labels[i])],
'predictions':predictions[i],
'wrong_classification':np.argmax(labels[i]) != np.argmax(predictions[i]),
'top5preds_idx' : top5,
'top5preds_values' : predictions[i][top5],
'top5preds_names': signnames[top5]
}
results.append(pred)
return results
def plot_pred_and_image(height,bars,image,img_name,title,save_images=False):
"""
Plot a 1x2 subplot with the top5 predictions and the
corresponding image
:param height: values of top5 predictions
:param bars: corresponding traffic signs
:param image: image to plot
:param img_name: name of the image (the traffic sign)
:param title: plot title
:param save_images: flag for image saving
:return: plots and save an image
"""
plt.figure(figsize=(3,3))
f,((ax1,ax2)) = plt.subplots(1,2,)
f.tight_layout(pad=5.0)
# create dataset
y_pos = np.arange(len(bars))
# Create horizontal bars
ax1.barh(y_pos, height)
# Create names on the x-axis
ax1.set_yticks(y_pos)
ax1.set_yticklabels(bars)
ax1.invert_yaxis() # labels read top-to-bottom
ax2.imshow(image,cmap='gray')
ax2.set_title(img_name)
f.suptitle(title, fontsize=16)
# Show graphic
plt.show()
if(save_images):
f.savefig('writeup_images/Wrong_pred_image_{}.png'.format(pred_index),
dpi=300,
bbox_inches = "tight")
def plot_test_images(
results,
show_correct_preds=False,
show_wrong_preds = False,
save_images=False):
"""
Organizes the plotting of predicted images and plots them
:param results: all predictions
:param show_correct_preds: flag for showing correct predictions of signs
:param show_wrong_preds: flag for showing wrong predictions of signs
:param save_images: flag for image saving
:return: plots and save an image
"""
for pred_index,pred in enumerate(results):
if (show_wrong_preds):
if pred['wrong_classification']:
print("Predicted class:", pred['name'])
print("Instead of: ", pred['correct_class'])
print("Certainty:", pred['certainty'])
print("Correct prediction:", not pred['wrong_classification'])
print("Top 5 predictions", pred['top5preds_names'])
print("Top 5 predictions values", pred['top5preds_values'])
#plot the results
plot_pred_and_image(
pred['top5preds_values'],
pred['top5preds_names'],
pred['image'],
pred['correct_class'],
'Wrong prediction',
save_images )
print("-------------------------------------\n")
elif(show_correct_preds):
if pred['wrong_classification'] == False:
print("Predicted class:", pred['name'])
print("Certainty:", pred['certainty'])
print("Correct prediction:", not pred['wrong_classification'])
print("Top 5 predictions", pred['top5preds_names'])
print("Top 5 predictions values", pred['top5preds_values'])
#plot the results
plot_pred_and_image(
pred['top5preds_values'],
pred['top5preds_names'],
pred['image'],
pred['correct_class'],
'Correct prediction',
save_images )
print("-------------------------------------\n")
### Run the predictions here and use the model to output the prediction for each image.
### Make sure to pre-process the images with the same pre-processing pipeline used earlier.
### Feel free to use as many code cells as needed.
# Evaluate the model on the test data using `evaluate`
results = loaded_model.evaluate(X_test,
y_test,
batch_size=128,
verbose=0)
print("test loss, test acc:", results)
results_own = loaded_model.evaluate(X_own_test,
y_own_test,
verbose=0)
print("Own test loss, Own test acc:", results_own)
test loss, test acc: [0.17693284153938293, 0.9609659314155579] Own test loss, Own test acc: [0.27811363339424133, 0.9444444179534912]
# Generate predictions (probabilities -- the output of the last layer)
# on new data using `predict`
print("Generate predictions for all samples")
predictions = loaded_model.predict(X_own_test,verbose=1)
details = pred_details(predictions,X_own_test,y_own_test,signnames)
Generate predictions for all samples 2/2 [==============================] - 0s 11ms/step
### Calculate the accuracy for the new images.
### For example, if the model predicted 1 out of 5 signs correctly,
### it's 20% accurate on these new images.
def eval_accuracy(results):
"""
Evaluate the prediction accuracy of my own images
:param results: all predictions
:return: plots and save an image
"""
result = {'total':len(results),
'correct':0,
'wrong':0,
'accuracy':0.0}
correct = 0.0
wrong = 0
for pred in results:
if(pred['wrong_classification']==False):
correct += 1
else:
wrong+=1
result['correct'] = correct
result['wrong'] = wrong
result['accuracy'] = correct/len(results)
return result
print(eval_accuracy(details))
{'total': 54, 'correct': 51.0, 'wrong': 3, 'accuracy': 0.9444444444444444}
### Print out the top five softmax probabilities for the predictions on the German traffic sign images found on the web.
plot_test_images(details,
show_correct_preds=True,
show_wrong_preds=False,
save_images=False)
Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'No entry' 'End of speed limit (80km/h)' 'Speed limit (100km/h)' 'Roundabout mandatory'] Top 5 predictions values [1.0000000e+00 1.9700419e-15 1.8342218e-16 2.2566294e-17 4.0773817e-18]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Keep right Certainty: 0.9922695 Correct prediction: True Top 5 predictions ['Keep right' 'Turn left ahead' 'Go straight or left' 'No entry' 'Go straight or right'] Top 5 predictions values [9.9226952e-01 7.1921796e-03 4.6006267e-04 3.8725608e-05 1.8978346e-05]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Roundabout mandatory Certainty: 0.9996617 Correct prediction: True Top 5 predictions ['Roundabout mandatory' 'Turn left ahead' 'Keep right' 'Speed limit (120km/h)' 'Vehicles over 3.5 metric tons prohibited'] Top 5 predictions values [9.9966168e-01 2.2072853e-04 4.8673264e-05 4.0549839e-05 2.1177562e-05]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Roundabout mandatory Certainty: 0.9999261 Correct prediction: True Top 5 predictions ['Roundabout mandatory' 'Turn left ahead' 'Speed limit (120km/h)' 'Vehicles over 3.5 metric tons prohibited' 'Keep right'] Top 5 predictions values [9.9992609e-01 4.6713114e-05 1.5399986e-05 4.8380912e-06 4.7025396e-06]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Keep right Certainty: 1.0 Correct prediction: True Top 5 predictions ['Keep right' 'Turn right ahead' 'No entry' 'Turn left ahead' 'Keep left'] Top 5 predictions values [1.0000000e+00 1.4754780e-09 8.5058760e-10 2.5373992e-10 4.2124790e-11]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Keep right Certainty: 1.0 Correct prediction: True Top 5 predictions ['Keep right' 'Turn left ahead' 'No entry' 'Priority road' 'Go straight or right'] Top 5 predictions values [1.0000000e+00 2.1748673e-12 6.6180958e-13 5.5602173e-15 3.3313979e-15]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Roundabout mandatory Certainty: 0.9940528 Correct prediction: True Top 5 predictions ['Roundabout mandatory' 'Speed limit (120km/h)' 'Keep right' 'Turn left ahead' 'Stop'] Top 5 predictions values [9.9405283e-01 2.5579846e-03 2.0292904e-03 1.0207802e-03 1.4443134e-04]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 1.0 Correct prediction: True Top 5 predictions ['Ahead only' 'Priority road' 'Yield' 'Turn left ahead' 'Go straight or left'] Top 5 predictions values [1.0000000e+00 2.6600214e-08 2.6679059e-09 1.4949363e-09 5.1465732e-10]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 0.99968827 Correct prediction: True Top 5 predictions ['Ahead only' 'Yield' 'Turn left ahead' 'General caution' 'Traffic signals'] Top 5 predictions values [9.9968827e-01 2.3671071e-04 2.6885004e-05 1.1121552e-05 9.4643283e-06]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Traffic signals Certainty: 0.99980146 Correct prediction: True Top 5 predictions ['Traffic signals' 'Pedestrians' 'Right-of-way at the next intersection' 'Dangerous curve to the right' 'Road work'] Top 5 predictions values [9.9980146e-01 1.2454635e-04 7.3347859e-05 2.3706654e-07 2.3087895e-07]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No passing Certainty: 0.9999999 Correct prediction: True Top 5 predictions ['No passing' 'Speed limit (50km/h)' 'Vehicles over 3.5 metric tons prohibited' 'No passing for vehicles over 3.5 metric tons' 'Speed limit (80km/h)'] Top 5 predictions values [9.9999988e-01 3.8147764e-08 1.9731964e-08 1.6567624e-08 5.3193419e-09]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 0.99999475 Correct prediction: True Top 5 predictions ['Ahead only' 'Priority road' 'Yield' 'No entry' 'Turn left ahead'] Top 5 predictions values [9.9999475e-01 4.7729100e-06 1.7015940e-07 1.2954183e-07 9.3237880e-08]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Yield Certainty: 0.99999964 Correct prediction: True Top 5 predictions ['Yield' 'Bumpy road' 'No vehicles' 'End of all speed and passing limits' 'Dangerous curve to the left'] Top 5 predictions values [9.9999964e-01 2.3810310e-07 1.3775109e-07 3.4093961e-08 8.7689012e-10]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 1.0 Correct prediction: True Top 5 predictions ['Ahead only' 'Turn left ahead' 'Priority road' 'Go straight or left' 'Yield'] Top 5 predictions values [1.0000000e+00 1.2460784e-11 1.2956097e-12 5.8152471e-14 4.5873043e-15]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No passing Certainty: 0.94645953 Correct prediction: True Top 5 predictions ['No passing' 'Speed limit (50km/h)' 'Speed limit (80km/h)' 'Speed limit (60km/h)' 'No passing for vehicles over 3.5 metric tons'] Top 5 predictions values [9.4645953e-01 4.2229947e-02 8.2454374e-03 2.2261746e-03 6.4758566e-04]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No entry Certainty: 0.9842152 Correct prediction: True Top 5 predictions ['No entry' 'Keep left' 'Ahead only' 'Priority road' 'End of no passing'] Top 5 predictions values [0.9842152 0.0035347 0.00255744 0.00185657 0.0017033 ]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No passing Certainty: 0.999632 Correct prediction: True Top 5 predictions ['No passing' 'Speed limit (50km/h)' 'No passing for vehicles over 3.5 metric tons' 'Speed limit (80km/h)' 'Speed limit (60km/h)'] Top 5 predictions values [9.9963200e-01 3.5976744e-04 5.0232229e-06 1.6761338e-06 1.3678080e-06]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Yield Certainty: 0.9999329 Correct prediction: True Top 5 predictions ['Yield' 'Bumpy road' 'Traffic signals' 'General caution' 'Dangerous curve to the left'] Top 5 predictions values [9.9993289e-01 2.5455121e-05 2.2322518e-05 1.8920698e-05 3.2291507e-07]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 1.0 Correct prediction: True Top 5 predictions ['Ahead only' 'General caution' 'Turn left ahead' 'End of all speed and passing limits' 'Go straight or left'] Top 5 predictions values [1.0000000e+00 8.5076612e-10 2.4581111e-11 6.5752325e-12 1.5635817e-12]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 1.0 Correct prediction: True Top 5 predictions ['Ahead only' 'Priority road' 'Road narrows on the right' 'Go straight or left' 'Right-of-way at the next intersection'] Top 5 predictions values [1.0000000e+00 1.3996943e-08 3.7389059e-10 3.6575853e-10 2.2238561e-10]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Yield Certainty: 1.0 Correct prediction: True Top 5 predictions ['Yield' 'Bumpy road' 'Priority road' 'End of all speed and passing limits' 'Dangerous curve to the left'] Top 5 predictions values [1.0000000e+00 7.3691875e-10 1.0611179e-10 4.5398902e-12 4.3990042e-12]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No passing Certainty: 0.47646752 Correct prediction: True Top 5 predictions ['No passing' 'No passing for vehicles over 3.5 metric tons' 'Speed limit (80km/h)' 'Speed limit (50km/h)' 'Speed limit (60km/h)'] Top 5 predictions values [0.47646752 0.23795351 0.17852016 0.0671119 0.00828112]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No entry Certainty: 0.9999858 Correct prediction: True Top 5 predictions ['No entry' 'End of all speed and passing limits' 'Priority road' 'End of no passing' 'Keep right'] Top 5 predictions values [9.9998581e-01 7.5067619e-06 5.3494527e-06 1.1469824e-06 1.3291373e-07]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Turn right ahead Certainty: 1.0 Correct prediction: True Top 5 predictions ['Turn right ahead' 'Ahead only' 'Go straight or right' 'Keep right' 'Stop'] Top 5 predictions values [1.0000000e+00 7.7443785e-10 1.2658757e-10 4.5059720e-11 2.1508099e-11]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No passing Certainty: 1.0 Correct prediction: True Top 5 predictions ['No passing' 'Speed limit (50km/h)' 'End of no passing' 'Bumpy road' 'End of all speed and passing limits'] Top 5 predictions values [1.0000000e+00 3.2813453e-11 3.1540867e-11 2.0727727e-12 1.9573804e-13]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 0.9077342 Correct prediction: True Top 5 predictions ['Ahead only' 'Pedestrians' 'Right-of-way at the next intersection' 'General caution' 'Road narrows on the right'] Top 5 predictions values [9.07734215e-01 8.99938196e-02 1.24883873e-03 9.06332396e-04 1.04801285e-04]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 0.99996364 Correct prediction: True Top 5 predictions ['Ahead only' 'Turn left ahead' 'Priority road' 'Keep right' 'Road narrows on the right'] Top 5 predictions values [9.9996364e-01 2.3947910e-05 1.0243150e-05 8.8406176e-07 2.2987383e-07]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No passing Certainty: 1.0 Correct prediction: True Top 5 predictions ['No passing' 'Speed limit (50km/h)' 'End of no passing' 'Bumpy road' 'Speed limit (120km/h)'] Top 5 predictions values [1.0000000e+00 2.2195941e-09 2.6446750e-10 4.1202593e-12 2.7778166e-12]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No entry Certainty: 0.99706405 Correct prediction: True Top 5 predictions ['No entry' 'End of no passing' 'Priority road' 'End of all speed and passing limits' 'End of no passing by vehicles over 3.5 metric tons'] Top 5 predictions values [9.9706405e-01 2.5563021e-03 3.1238477e-04 5.2737058e-05 1.3822052e-05]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No entry Certainty: 0.9994261 Correct prediction: True Top 5 predictions ['No entry' 'Keep right' 'Priority road' 'Keep left' 'End of all speed and passing limits'] Top 5 predictions values [9.9942613e-01 5.4754579e-04 1.3501532e-05 5.8158867e-06 4.9719451e-06]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No passing Certainty: 0.9999999 Correct prediction: True Top 5 predictions ['No passing' 'Speed limit (50km/h)' 'End of no passing' 'No vehicles' 'Speed limit (30km/h)'] Top 5 predictions values [9.9999988e-01 8.1536101e-08 5.0683843e-11 8.0724932e-12 1.7841919e-12]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 0.999884 Correct prediction: True Top 5 predictions ['Ahead only' 'General caution' 'Pedestrians' 'Right-of-way at the next intersection' 'Go straight or left'] Top 5 predictions values [9.9988401e-01 7.0844661e-05 2.9636418e-05 1.3484614e-05 1.4387538e-06]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Certainty: 0.9999999 Correct prediction: True Top 5 predictions ['Ahead only' 'Priority road' 'No entry' 'Yield' 'Go straight or left'] Top 5 predictions values [9.9999988e-01 8.1837428e-08 3.4253906e-09 2.2545259e-09 1.3079741e-09]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: No passing Certainty: 1.0 Correct prediction: True Top 5 predictions ['No passing' 'End of no passing' 'Speed limit (50km/h)' 'No passing for vehicles over 3.5 metric tons' 'Vehicles over 3.5 metric tons prohibited'] Top 5 predictions values [1.0000000e+00 1.6156085e-11 7.6307502e-12 2.6160307e-12 3.7177353e-14]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Go straight or right Certainty: 0.99998474 Correct prediction: True Top 5 predictions ['Go straight or right' 'Turn right ahead' 'Ahead only' 'Keep right' 'Priority road'] Top 5 predictions values [9.9998474e-01 1.0575796e-05 4.6468604e-06 8.5352632e-09 2.2550164e-09]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'No entry' 'Roundabout mandatory' 'Speed limit (100km/h)' 'End of speed limit (80km/h)'] Top 5 predictions values [1.0000000e+00 1.1910654e-12 4.5550425e-16 1.0029985e-16 5.1431202e-17]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Turn left ahead Certainty: 0.5729064 Correct prediction: True Top 5 predictions ['Turn left ahead' 'Road work' 'Go straight or left' 'Road narrows on the right' 'Ahead only'] Top 5 predictions values [0.5729064 0.21166202 0.07337746 0.032849 0.02800057]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Turn left ahead Certainty: 0.78169954 Correct prediction: True Top 5 predictions ['Turn left ahead' 'Dangerous curve to the left' 'Go straight or left' 'Road work' 'Speed limit (70km/h)'] Top 5 predictions values [0.78169954 0.20904577 0.00319625 0.00178346 0.00172135]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'Speed limit (100km/h)' 'No entry' 'Roundabout mandatory' 'Double curve'] Top 5 predictions values [1.0000000e+00 8.9658159e-10 1.3267706e-10 5.9722873e-12 1.3669677e-12]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Go straight or right Certainty: 0.9999943 Correct prediction: True Top 5 predictions ['Go straight or right' 'Ahead only' 'Turn right ahead' 'Keep right' 'Go straight or left'] Top 5 predictions values [9.9999428e-01 2.6637356e-06 2.3424045e-06 6.7235806e-07 4.0626293e-08]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Go straight or right Certainty: 0.99999607 Correct prediction: True Top 5 predictions ['Go straight or right' 'Ahead only' 'Turn right ahead' 'Keep right' 'Priority road'] Top 5 predictions values [9.9999607e-01 2.1178857e-06 1.7133523e-06 8.6566899e-08 1.7786631e-09]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Turn left ahead Certainty: 0.9846087 Correct prediction: True Top 5 predictions ['Turn left ahead' 'No vehicles' 'Speed limit (70km/h)' 'Speed limit (30km/h)' 'Ahead only'] Top 5 predictions values [9.8460871e-01 7.4111065e-03 6.4631295e-03 4.9488037e-04 3.7675325e-04]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'Speed limit (100km/h)' 'No entry' 'End of speed limit (80km/h)' 'Roundabout mandatory'] Top 5 predictions values [1.0000000e+00 1.5067897e-10 6.0827808e-11 1.0578780e-11 1.5785976e-12]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'No entry' 'Speed limit (100km/h)' 'End of speed limit (80km/h)' 'Yield'] Top 5 predictions values [1.0000000e+00 1.6758714e-10 4.2344687e-12 1.3651647e-12 1.3482631e-14]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Turn left ahead Certainty: 0.9985864 Correct prediction: True Top 5 predictions ['Turn left ahead' 'Ahead only' 'Keep right' 'Go straight or left' 'Stop'] Top 5 predictions values [9.9858642e-01 1.2027619e-03 7.7625198e-05 5.3644679e-05 3.6993672e-05]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Go straight or right Certainty: 0.53665847 Correct prediction: True Top 5 predictions ['Go straight or right' 'Keep right' 'Turn right ahead' 'Ahead only' 'Keep left'] Top 5 predictions values [5.3665847e-01 3.8946488e-01 7.1189836e-02 2.6642748e-03 1.3724645e-05]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'No entry' 'Speed limit (100km/h)' 'End of speed limit (80km/h)' 'Keep right'] Top 5 predictions values [1.0000000e+00 1.9506023e-08 1.2894771e-10 7.9462348e-12 3.5139480e-12]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'No entry' 'End of speed limit (80km/h)' 'Speed limit (100km/h)' 'End of no passing by vehicles over 3.5 metric tons'] Top 5 predictions values [1.0000000e+00 3.2681318e-13 1.4065887e-15 3.1663627e-16 1.7937887e-18]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Turn left ahead Certainty: 0.9990901 Correct prediction: True Top 5 predictions ['Turn left ahead' 'Keep right' 'Go straight or left' 'Roundabout mandatory' 'Ahead only'] Top 5 predictions values [9.9909008e-01 4.1169842e-04 3.7695337e-04 9.7177224e-05 2.1854277e-05]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'No entry' 'Speed limit (100km/h)' 'End of speed limit (80km/h)' 'Roundabout mandatory'] Top 5 predictions values [1.0000000e+00 2.8949542e-11 4.8634568e-14 3.3314060e-14 2.5525766e-15]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Priority road Certainty: 1.0 Correct prediction: True Top 5 predictions ['Priority road' 'No entry' 'End of speed limit (80km/h)' 'Speed limit (100km/h)' 'Roundabout mandatory'] Top 5 predictions values [1.0000000e+00 3.1353949e-12 2.2720468e-14 1.0697462e-14 5.1716130e-15]
<Figure size 300x300 with 0 Axes>
-------------------------------------
#plot wrongly predicted images
plot_test_images(details,
show_correct_preds=False,
show_wrong_preds=True,
save_images=False)
Predicted class: Priority road Instead of: No entry Certainty: 0.9837916 Correct prediction: False Top 5 predictions ['Priority road' 'No entry' 'End of no passing by vehicles over 3.5 metric tons' 'End of no passing' 'Keep right'] Top 5 predictions values [9.8379159e-01 1.6207857e-02 1.8546235e-07 1.7226644e-07 6.1256536e-08]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: End of all speed and passing limits Instead of: No entry Certainty: 0.54557186 Correct prediction: False Top 5 predictions ['End of all speed and passing limits' 'No vehicles' 'Yield' 'End of no passing' 'Priority road'] Top 5 predictions values [0.54557186 0.17437021 0.10472484 0.03869934 0.02921509]
<Figure size 300x300 with 0 Axes>
------------------------------------- Predicted class: Ahead only Instead of: Go straight or right Certainty: 0.9520398 Correct prediction: False Top 5 predictions ['Ahead only' 'Go straight or right' 'Keep right' 'Turn right ahead' 'Priority road'] Top 5 predictions values [9.5203978e-01 3.0644638e-02 1.6842106e-02 4.6105051e-04 7.7392333e-06]
<Figure size 300x300 with 0 Axes>
-------------------------------------
Note: Once you have completed all of the code implementations and successfully answered each question above, you may finalize your work by exporting the iPython Notebook as an HTML document. You can do this by using the menu above and navigating to \n", "File -> Download as -> HTML (.html). Include the finished document along with this notebook as your submission.
This Section is not required to complete but acts as an additional excersise for understaning the output of a neural network's weights. While neural networks can be a great learning device they are often referred to as a black box. We can understand what the weights of a neural network look like better by plotting their feature maps. After successfully training your neural network you can see what it's feature maps look like by plotting the output of the network's weight layers in response to a test stimuli image. From these plotted feature maps, it's possible to see what characteristics of an image the network finds interesting. For a sign, maybe the inner network feature maps react with high activation to the sign's boundary outline or to the contrast in the sign's painted symbol.
Provided for you below is the function code that allows you to get the visualization output of any tensorflow weight layer you want. The inputs to the function should be a stimuli image, one used during training or a new one you provided, and then the tensorflow variable name that represents the layer's state during the training process, for instance if you wanted to see what the LeNet lab's feature maps looked like for it's second convolutional layer you could enter conv2 as the tf_activation variable.
For an example of what feature map outputs look like, check out NVIDIA's results in their paper End-to-End Deep Learning for Self-Driving Cars in the section Visualization of internal CNN State. NVIDIA was able to show that their network's inner weights had high activations to road boundary lines by comparing feature maps from an image with a clear path to one without. Try experimenting with a similar test to show that your trained network's weights are looking for interesting features, whether it's looking at differences in feature maps from images with or without a sign, or even what feature maps look like in a trained network vs a completely untrained one on the same sign image.
Your output should look something like this (above)
### Visualize your network's feature maps here.
### Feel free to use as many code cells as needed.
# image_input: the test image being fed into the network to produce the feature maps
# tf_activation: should be a tf variable name used during your training procedure that represents the calculated state of a specific weight layer
# activation_min/max: can be used to view the activation contrast in more detail, by default matplot sets min and max to the actual min and max values of the output
# plt_num: used to plot out multiple different weight feature map sets on the same block, just extend the plt number for each new feature map entry
def outputFeatureMap(image_input, tf_activation, activation_min=-1, activation_max=-1 ,plt_num=1):
# Here make sure to preprocess your image_input in a way your network expects
# with size, normalisation, ect if needed
# image_input =
# Note: x should be the same name as your network's tensorflow data placeholder variable
# If you get an error tf_activation is not defined it may be having trouble accessing the variable from inside a function
activation = tf_activation.eval(session=sess,feed_dict={x : image_input})
featuremaps = activation.shape[3]
plt.figure(plt_num, figsize=(15,15))
for featuremap in range(featuremaps):
plt.subplot(6,8, featuremap+1) # sets the number of feature maps to show on each row and column
plt.title('FeatureMap ' + str(featuremap)) # displays the feature map number
if activation_min != -1 & activation_max != -1:
plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmin =activation_min, vmax=activation_max, cmap="gray")
elif activation_max != -1:
plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmax=activation_max, cmap="gray")
elif activation_min !=-1:
plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmin=activation_min, cmap="gray")
else:
plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", cmap="gray")